{
 "cells": [
  {
   "cell_type": "code",
   "id": "initial_id",
   "metadata": {
    "collapsed": true,
    "ExecuteTime": {
     "end_time": "2025-03-06T04:02:28.881087Z",
     "start_time": "2025-03-06T04:02:25.273129Z"
    }
   },
   "source": [
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import numpy as np\n",
    "import sklearn\n",
    "import pandas as pd\n",
    "import os\n",
    "import sys\n",
    "import time\n",
    "from tqdm.auto import tqdm\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "print(sys.version_info)\n",
    "for module in mpl, np, pd, sklearn, torch:\n",
    "    print(module.__name__, module.__version__)\n",
    "\n",
    "device = torch.device(\"cuda:0\") if torch.cuda.is_available() else torch.device(\"cpu\")\n",
    "print(device)\n",
    "\n",
    "seed = 42\n",
    "torch.manual_seed(seed)\n",
    "torch.cuda.manual_seed_all(seed)"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sys.version_info(major=3, minor=12, micro=3, releaselevel='final', serial=0)\n",
      "matplotlib 3.10.0\n",
      "numpy 1.26.4\n",
      "pandas 2.2.3\n",
      "sklearn 1.6.0\n",
      "torch 2.3.1+cu121\n",
      "cuda:0\n"
     ]
    }
   ],
   "execution_count": 1
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "# 数据准备",
   "id": "1259b84ca1aa2197"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-06T04:04:43.606798Z",
     "start_time": "2025-03-06T04:04:43.602210Z"
    }
   },
   "cell_type": "code",
   "source": [
    "with open(\"./shakespeare.txt\", \"r\", encoding=\"utf8\") as file:\n",
    "    text = file.read()\n",
    "\n",
    "print(\"length\", len(text))\n",
    "print(text[0:100])"
   ],
   "id": "799899c4cf5217b",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "length 1115394\n",
      "First Citizen:\n",
      "Before we proceed any further, hear me speak.\n",
      "\n",
      "All:\n",
      "Speak, speak.\n",
      "\n",
      "First Citizen:\n",
      "You\n"
     ]
    }
   ],
   "execution_count": 3
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "## 构建字典",
   "id": "7132820f089e1460"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-06T04:11:31.422304Z",
     "start_time": "2025-03-06T04:11:31.354855Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 1. 建立字典\n",
    "\n",
    "#去重，留下独立字符，并排序\n",
    "vocab = sorted(set(text))\n",
    "print(f'字典大小：{len(vocab)}')\n",
    "print(f'字典：\\n{vocab}')\n",
    "\n",
    "# 2. 建立id_to_vocab 和 vocab_to_id\n",
    "\n",
    "#每个字符都编好号，enumerate对每一个位置编号，生成的是列表中是元组\n",
    "char2idx = {char: idx for idx, char in enumerate(vocab)}\n",
    "print(f'char2idx：\\n{char2idx}')\n",
    "\n",
    "# 把vocab从列表变为ndarray\n",
    "idx2char = np.array(vocab)\n",
    "\n",
    "# 3. 将数据转换为id序列\n",
    "\n",
    "#把字符都转换为id\n",
    "text_as_int = np.array([char2idx[c] for c in text])\n",
    "print(f'文本长度：{text_as_int.shape}')"
   ],
   "id": "55d24dbc34164bdb",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "字典大小：65\n",
      "字典：\n",
      "['\\n', ' ', '!', '$', '&', \"'\", ',', '-', '.', '3', ':', ';', '?', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z']\n",
      "char2idx：\n",
      "{'\\n': 0, ' ': 1, '!': 2, '$': 3, '&': 4, \"'\": 5, ',': 6, '-': 7, '.': 8, '3': 9, ':': 10, ';': 11, '?': 12, 'A': 13, 'B': 14, 'C': 15, 'D': 16, 'E': 17, 'F': 18, 'G': 19, 'H': 20, 'I': 21, 'J': 22, 'K': 23, 'L': 24, 'M': 25, 'N': 26, 'O': 27, 'P': 28, 'Q': 29, 'R': 30, 'S': 31, 'T': 32, 'U': 33, 'V': 34, 'W': 35, 'X': 36, 'Y': 37, 'Z': 38, 'a': 39, 'b': 40, 'c': 41, 'd': 42, 'e': 43, 'f': 44, 'g': 45, 'h': 46, 'i': 47, 'j': 48, 'k': 49, 'l': 50, 'm': 51, 'n': 52, 'o': 53, 'p': 54, 'q': 55, 'r': 56, 's': 57, 't': 58, 'u': 59, 'v': 60, 'w': 61, 'x': 62, 'y': 63, 'z': 64}\n",
      "文本长度：(1115394,)\n"
     ]
    }
   ],
   "execution_count": 12
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "## 把莎士比亚文集分成一个一个的样本",
   "id": "3b50cca33b371905"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-06T04:33:03.662219Z",
     "start_time": "2025-03-06T04:33:03.656706Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from torch.utils.data import Dataset, DataLoader\n",
    "\n",
    "\n",
    "class CharDataset(Dataset):\n",
    "\n",
    "    def __init__(self, text_as_int, seq_length):\n",
    "        self.text_as_int = text_as_int\n",
    "        self.sub_len = seq_length + 1  #一个样本的长度\n",
    "        self.num_seq = len(text_as_int) // self.sub_len  #样本的个数\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        # index是样本的索引，返回的是一个样本，比如第一个，就是0-100的字符,总计101个字符\n",
    "        return self.text_as_int[index * self.sub_len: (index + 1) * self.sub_len]\n",
    "\n",
    "    def __len__(self):\n",
    "        # 返回样本的个数\n",
    "        return self.num_seq\n",
    "\n",
    "\n",
    "# 定义一个函数，把一个batch的样本转换为输入和输出，输入是前100个字符，输出是后100个字符\n",
    "def collat_fct(batch):\n",
    "    src_list = []  #输入\n",
    "    trg_list = []  #输出\n",
    "    for part in batch:\n",
    "        src_list.append(part[:-1])  #输入\n",
    "        trg_list.append(part[1:])  #输出\n",
    "\n",
    "    src_list = np.array(src_list)  #把列表转换为ndarray\n",
    "    trg_list = np.array(trg_list)  #把列表转换为ndarray\n",
    "    return torch.Tensor(src_list).to(dtype=torch.int64), torch.Tensor(trg_list).to(\n",
    "        dtype=torch.int64)  #返回的是一个元组，元组中的每一个元素是一个torch.Tensor\n",
    "\n",
    "\n",
    "#每个样本的长度是101，也就是100个字符+1个结束符\n",
    "train_ds = CharDataset(text_as_int, 100)\n",
    "train_dl = DataLoader(train_ds, batch_size=64, shuffle=True, collate_fn=collat_fct)"
   ],
   "id": "10e666e84bf10bca",
   "outputs": [],
   "execution_count": 13
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "# 模型定义",
   "id": "f1378a9a665523e6"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-06T04:38:29.029987Z",
     "start_time": "2025-03-06T04:38:29.019501Z"
    }
   },
   "cell_type": "code",
   "source": [
    "class CharRNN(nn.Module):\n",
    "    def __init__(self, vocab_size, embedding_dim=256, hidden_dim=1024):\n",
    "        super(CharRNN, self).__init__()\n",
    "        self.embedding = nn.Embedding(vocab_size, embedding_dim)\n",
    "        #batch_first=True,输入的数据格式是(batch_size, seq_len, embedding_dim)\n",
    "        self.rnn = nn.RNN(embedding_dim, hidden_dim, batch_first=True)\n",
    "        self.fc = nn.Linear(hidden_dim, vocab_size)\n",
    "\n",
    "    def forward(self, x, hidden=None):\n",
    "        x = self.embedding(x)  #(batch_size, seq_len) -> (batch_size, seq_len, embedding_dim) (64, 100, 256)\n",
    "        #这里和02的差异是没有只拿最后一个输出，而是把所有的输出都拿出来了\n",
    "        #(batch_size, seq_len, embedding_dim)->(batch_size, seq_len, hidden_dim)(64, 100, 1024)\n",
    "        output, hidden = self.rnn(x, hidden)\n",
    "        x = self.fc(output)  #[bs, seq_len, hidden_dim]--->[bs, seq_len, vocab_size] (64, 100,65)\n",
    "        return x, hidden  #x的shape是(batch_size, seq_len, vocab_size)\n",
    "\n",
    "\n",
    "vocab_size = len(vocab)\n",
    "\n",
    "print(\"{:=^80}\".format(\" 一层单向 RNN \"))\n",
    "for key, value in CharRNN(vocab_size).named_parameters():\n",
    "    print(f\"{key:^40}paramerters num: {np.prod(value.shape)}\")"
   ],
   "id": "ef6df406c8e17e0e",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=================================== 一层单向 RNN ===================================\n",
      "            embedding.weight            paramerters num: 16640\n",
      "            rnn.weight_ih_l0            paramerters num: 262144\n",
      "            rnn.weight_hh_l0            paramerters num: 1048576\n",
      "             rnn.bias_ih_l0             paramerters num: 1024\n",
      "             rnn.bias_hh_l0             paramerters num: 1024\n",
      "               fc.weight                paramerters num: 66560\n",
      "                fc.bias                 paramerters num: 65\n"
     ]
    }
   ],
   "execution_count": 14
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "# 模型训练",
   "id": "3c8535b452623b47"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-06T04:39:18.793792Z",
     "start_time": "2025-03-06T04:39:18.788296Z"
    }
   },
   "cell_type": "code",
   "source": [
    "class SaveCheckpointsCallback:\n",
    "    def __init__(self, save_dir, save_step=5000, save_best_only=True):\n",
    "        \"\"\"\n",
    "        Save checkpoints each save_epoch epoch. \n",
    "        We save checkpoint by epoch in this implementation.\n",
    "        Usually, training scripts with pytorch evaluating model and save checkpoint by step.\n",
    "\n",
    "        Args:\n",
    "            save_dir (str): dir to save checkpoint\n",
    "            save_epoch (int, optional): the frequency to save checkpoint. Defaults to 1.\n",
    "            save_best_only (bool, optional): If True, only save the best model or save each model at every epoch.\n",
    "        \"\"\"\n",
    "        self.save_dir = save_dir\n",
    "        self.save_step = save_step\n",
    "        self.save_best_only = save_best_only\n",
    "        self.best_metrics = -1\n",
    "\n",
    "        # mkdir\n",
    "        if not os.path.exists(self.save_dir):\n",
    "            os.mkdir(self.save_dir)\n",
    "\n",
    "    def __call__(self, step, state_dict, metric=None):\n",
    "        if step % self.save_step > 0:\n",
    "            return\n",
    "\n",
    "        if self.save_best_only:\n",
    "            assert metric is not None\n",
    "            if metric >= self.best_metrics:\n",
    "                # save checkpoints\n",
    "                torch.save(state_dict, os.path.join(self.save_dir, \"best.ckpt\"))\n",
    "                # update best metrics\n",
    "                self.best_metrics = metric\n",
    "        else:\n",
    "            torch.save(state_dict, os.path.join(self.save_dir, f\"{step}.ckpt\"))\n",
    "\n"
   ],
   "id": "31e50231b23dd9e2",
   "outputs": [],
   "execution_count": 15
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-06T04:39:34.449263Z",
     "start_time": "2025-03-06T04:39:34.442876Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 训练\n",
    "def training(\n",
    "        model,\n",
    "        train_loader,\n",
    "        epoch,\n",
    "        loss_fct,\n",
    "        optimizer,\n",
    "        save_ckpt_callback=None,\n",
    "        stateful=False  # 想用stateful，batch里的数据就必须连续，不能打乱\n",
    "):\n",
    "    record_dict = {\n",
    "        \"train\": [],\n",
    "    }\n",
    "\n",
    "    global_step = 0\n",
    "    model.train()\n",
    "    hidden = None\n",
    "    with tqdm(total=epoch * len(train_loader)) as pbar:\n",
    "        for epoch_id in range(epoch):\n",
    "            # training\n",
    "            for datas, labels in train_loader:\n",
    "                datas = datas.to(device)\n",
    "                labels = labels.to(device)\n",
    "                # 梯度清空\n",
    "                optimizer.zero_grad()\n",
    "                # 模型前向计算,如果数据集打乱了，stateful=False，hidden就要清空\n",
    "                # 如果数据集没有打乱，stateful=True，hidden就不需要清空\n",
    "                logits, hidden = model(datas, hidden=hidden if stateful else None)\n",
    "                # 计算损失,交叉熵损失第一个参数要是二阶张量，第二个参数要是一阶张量，所以要reshape\n",
    "                loss = loss_fct(logits.reshape(-1, vocab_size), labels.reshape(-1))\n",
    "                # 梯度回传\n",
    "                loss.backward()\n",
    "                # 调整优化器，包括学习率的变动等\n",
    "                optimizer.step()\n",
    "\n",
    "                loss = loss.cpu().item()\n",
    "                # record\n",
    "\n",
    "                record_dict[\"train\"].append({\n",
    "                    \"loss\": loss, \"step\": global_step\n",
    "                })\n",
    "\n",
    "                # 保存模型权重 save model checkpoint\n",
    "                if save_ckpt_callback is not None:\n",
    "                    save_ckpt_callback(global_step, model.state_dict(), metric=-loss)\n",
    "                # udate step\n",
    "                global_step += 1\n",
    "                pbar.update(1)\n",
    "                pbar.set_postfix({\"epoch\": epoch_id})\n",
    "\n",
    "    return record_dict"
   ],
   "id": "5368bb7e35017d93",
   "outputs": [],
   "execution_count": 16
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-06T04:39:52.398500Z",
     "start_time": "2025-03-06T04:39:51.091278Z"
    }
   },
   "cell_type": "code",
   "source": [
    "epoch = 100\n",
    "\n",
    "model = CharRNN(vocab_size=vocab_size)\n",
    "\n",
    "# 1. 定义损失函数 采用交叉熵损失 \n",
    "loss_fct = nn.CrossEntropyLoss()\n",
    "# 2. 定义优化器 采用 adam\n",
    "# Optimizers specified in the torch.optim package\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=0.001)\n",
    "\n",
    "# save best\n",
    "if not os.path.exists(\"checkpoints\"):\n",
    "    os.makedirs(\"checkpoints\")\n",
    "save_ckpt_callback = SaveCheckpointsCallback(\"checkpoints/text_generation\", save_step=1000, save_best_only=True)\n",
    "\n",
    "model = model.to(device)"
   ],
   "id": "dca1675c39da31f1",
   "outputs": [],
   "execution_count": 17
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-06T04:42:47.598788Z",
     "start_time": "2025-03-06T04:39:58.676103Z"
    }
   },
   "cell_type": "code",
   "source": [
    "record = training(\n",
    "    model,\n",
    "    train_dl,\n",
    "    epoch,\n",
    "    loss_fct,\n",
    "    optimizer,\n",
    "    save_ckpt_callback=save_ckpt_callback,\n",
    ")"
   ],
   "id": "df8f4d07d436b0d3",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "  0%|          | 0/17300 [00:00<?, ?it/s]"
      ],
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "444fa6aef556407687b1ea8e9426a01a"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "execution_count": 18
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-06T04:42:47.664102Z",
     "start_time": "2025-03-06T04:42:47.599792Z"
    }
   },
   "cell_type": "code",
   "source": [
    "plt.plot([i[\"step\"] for i in record[\"train\"][::50]], [i[\"loss\"] for i in record[\"train\"][::50]], label=\"train\")\n",
    "plt.grid()\n",
    "plt.show()"
   ],
   "id": "bba50900ba45dae2",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ],
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAikAAAGdCAYAAADXIOPgAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAUDpJREFUeJzt3Qd8VFX2wPGThJAQIBBaAqTQe5EqRZrSWRV1cQVX7G1x1cXVXdTdBV2FtS7/xUVdCzZEUcGGSgeR3jvSQ0lCT4CQQjL/z7kzbzITkkAgkJe83/fzGSZTeXPz8t6Ze889N8DlcrkEAADAZgKLewMAAADyQpACAABsiSAFAADYEkEKAACwJYIUAABgSwQpAADAlghSAACALRGkAAAAWyojJUB2drYcPHhQKlasKAEBAcW9OQAA4AJovdiTJ09KrVq1JDAwsHQGKRqgxMTEFPdmAACAi7Bv3z6Jjo4unUGK9qBYHzI8PLzI3jczM1Nmzpwpffv2leDgYHEq2oE2ULSBG+1AGyjaoGjaISUlxXQyWOfxUhmkWEM8GqAUdZASFhZm3tPpO6HT24E2oA0stANtoGiDom2Hi03VIHEWAADYEkEKAACwJYIUAABgSwQpAADAlghSAACALRGkAAAAWyJIAQAAtkSQAgAAbIkgBQAA2BJBCgAAsCWCFAAAYEsEKQAAwJYcHaS8v3ivfLk7ULYlnizuTQEAALk4OkiZsTFRFiYGyr7jZ4p7UwAAQC6ODlKshaOzXa5i3hIAAJCbo4OUwAB3mEKMAgCA/Tg6SPHEKPSkAABgQw4PUuhJAQDArhwdpAR6elKIUQAAsB9HBykkzgIAYF+ODlJInAUAwL4cHaRYXSkuohQAAGzH0UGKtyeluDcEAACcw9FBCjkpAADYl6ODFHJSAACwL0cHKVZXSjZBCgAAtuPoIMVbJ4WuFAAAbMfhQQqJswAA2JWjgxQSZwEAKKVByrhx48z6N48//niBz5s6dao0adJEQkNDpWXLljJjxgyxA9buAQCgFAYpK1askLfeektatWpV4PMWL14sQ4cOlXvvvVfWrFkjgwcPNpeNGzeKXVZBJicFAIBSEqScOnVKbr/9dvnf//4nERERBT53/Pjx0r9/f3nyySeladOm8vzzz0vbtm1lwoQJUtzISQEAwL7KXMyLRowYIYMGDZLevXvLP//5zwKfu2TJEhk5cqTfff369ZPp06fn+5r09HRzsaSkpJjrzMxMcykqLle2+33PZhXp+5Y01menDWgD32unoh1oA0UbFE07XGr7FTpImTJliqxevdoM91yIxMREiYyM9LtPb+v9+Rk7dqyMGTPmnPtnzpwpYWFhUlQOJWlHUqBs3bpVZiRvEaebNWuWOB1tQBtYaAfaQNEGl9YOqampcsWClH379sljjz1mNlaTYC+XUaNG+fW+aE9KTEyM9O3bV8LDw4vs//kpZa2sOXpIGjVqLAO71ROn0khXf6d9+vSR4OBgcSLagDaw0A60gaINiqYdrJGQKxKkrFq1Sg4dOmRySixZWVmycOFCk2OiQzRBQUF+r4mKipKkpCS/+/S23p+fkJAQc8lNG6god5agIHdKTmBQkKN3wsvVviURbUAbWGgH2kDRBpfWDpfadoVKnL3uuutkw4YNsnbtWu+lffv2JolWf84doKjOnTvLnDlz/O7TqEzvt0viLHVSAACwn0L1pFSsWFFatGjhd1/58uWlatWq3vuHDx8utWvXNnklSoeHevToIa+++qpJttWclpUrV8rbb78txY1ibgAAOKjibHx8vCQkJHhvd+nSRSZPnmyCktatW8sXX3xhZvbkDnaKQ4Bn8R5iFAAASskUZF/z588v8LYaMmSIudiN1ZNCkAIAgP04eu0ebzE3ohQAAGzH0UGKVRY/mxgFAADbcXSQ4klJoSw+AAA25OggxcpKYXYPAAD24+ggxepJoSsFAAD7cXSQkpOTQpQCAIDdODpI8c7uKe4NAQAA53B0kBJAWXwAAGzL2UGK55oYBQAA+3F0kOKdgkyQAgCA7Tg6SGG4BwAA+3J0kEIxNwAA7MvRQYqFtXsAALAfRwcpOQsMFveWAACA3BwdpFDMDQAA+3J0kEIxNwAA7MvRQYpVJyWbKAUAANtxdpDizUkhSgEAwG4cHaRQzA0AAPtydJBC4iwAAPbl8CCFxFkAAOzK2UGK55qcFAAA7MfRQQrF3AAAsC9HByk5OSnFvSUAACA3RwcpOcXciFIAALAbRwcpFnpSAACwH0cHKYHWpycpBQAA23F2kOIZ7qEnBQAA+3F0kJKzdg9RCgAAduPsIIUpyAAA2JbDgxT3NUEKAAD24+gghSnIAADYl6ODlJyclGLeEAAAcA5HBymB3uEeohQAAOzG0UGKlZRCTwoAAPbj6CDF6kkBAAD24+ggJcCTlUKdFAAA7MfRQUpOTkpxbwkAAMjN0UGKVcyNnhQAAOzH4UGK+5oYBQCAEh6kTJw4UVq1aiXh4eHm0rlzZ/nhhx/yff6kSZNMb4XvJTQ0VGw33EMxNwAAbKdMYZ4cHR0t48aNk4YNG5raIh988IHceOONsmbNGmnevHmer9FgZtu2becMsdgrcba4twQAAFxSkHL99df73X7hhRdM78rSpUvzDVI0KImKihI7InEWAIBSEqT4ysrKkqlTp8rp06fNsE9+Tp06JXFxcZKdnS1t27aVF198Md+AxpKenm4ulpSUFHOdmZlpLkUlKzvLc51dpO9b0lifnTagDXyvnYp2oA0UbVA07XCp7RfgKmRN+A0bNpigJC0tTSpUqCCTJ0+WgQMH5vncJUuWyPbt200eS3JysrzyyiuycOFC2bRpkxk6ys/o0aNlzJgx59yv/1dYWJgUlVVHAuTD7UHSMDxbHmmeXWTvCwAARFJTU2XYsGEmBtD0j8sepGRkZEh8fLz5D7/44gt55513ZMGCBdKsWbMLiqiaNm0qQ4cOleeff75QPSkxMTFy5MiRi/qQ+fl6zX7581ebpWNcZfnkvo7iVPp7mTVrlvTp00eCg4PFiWgD2sBCO9AGijYomnbQ83e1atUuOkgp9HBP2bJlpUGDBubndu3ayYoVK2T8+PHy1ltvnfe1+gHbtGkjO3bsKPB5ISEh5pLX64tyZwku4/n4AQGO3gkvV/uWRLQBbWChHWgDRRtcWjtcattdcp0UzTXx7fU4Xx6LDhfVrFlT7MCaaEQxNwAA7KdQPSmjRo2SAQMGSGxsrJw8edLkiMyfP19++ukn8/jw4cOldu3aMnbsWHP7ueeek06dOpmelxMnTsjLL78se/fulfvuu0/swJoOTYwCAEAJD1IOHTpkApGEhASpVKmSSYjVAEXHqpTmqgQG5nTOHD9+XO6//35JTEyUiIgIMzy0ePHiC8pfubLF3AAAQIkOUt59990CH9deFV+vv/66udgVqyADAGBfjl67h2JuAADYl6ODFE9HiinxDwAA7MXRQUqglThb3BsCAADO4egghSnIAADYl6ODFG9PCjEKAAC24+ggxdORItkEKQAA2I6zgxRrvIeuFAAAbMfhQYr7mp4UAADsx9FBilUnhcRZAADsx+FBClOQAQCwK0cHKRaKuQEAYD+ODlKYggwAgH05OkghcRYAAPtydJCSk5NClAIAgN04OkihmBsAAPbl7CDFilJISgEAwHYcHqS4oxR6UgAAsB9HBylWMTdiFAAA7MfhQYrVk0KYAgCA3Tg6SLEQowAAYD+ODlJyirkRpQAAYDeODlIo5gYAgH05OkjJSZwlSgEAwG4cHaQEeMq5MdoDAID9ODtIsXpSCFIAALAdhwcpTEEGAMCuHB2kUMwNAAD7cnSQkjO7hzAFAAC7cXiQQuIsAAB25eggxTvcQ5QCAIDtODpIsaYgU8wNAAD7cXSQQuIsAAD25eggJScnhTAFAAC7cXiQ4r4mRgEAwH6cHaR4rpmCDACA/Tg6SAm0hnuKe0MAAMA5HB2kMNwDAIB9OTxIsQZ8SJ4FAMBunB2k+PxMrRQAAOzF0UGKlZOiSJ4FAMBeHB6k5PxMjAIAQAkOUiZOnCitWrWS8PBwc+ncubP88MMPBb5m6tSp0qRJEwkNDZWWLVvKjBkzxC58OlLoSQEAoCQHKdHR0TJu3DhZtWqVrFy5Uq699lq58cYbZdOmTXk+f/HixTJ06FC59957Zc2aNTJ48GBz2bhxo9gtcRYAAJTgIOX666+XgQMHSsOGDaVRo0bywgsvSIUKFWTp0qV5Pn/8+PHSv39/efLJJ6Vp06by/PPPS9u2bWXChAliv8RZelIAALCTMhf7wqysLDOUc/r0aTPsk5clS5bIyJEj/e7r16+fTJ8+vcD3Tk9PNxdLSkqKuc7MzDSXopKVddb7c0ZGpgQHODNQsdq0KNu2pKENaAML7UAbKNqgaNrhUtuv0EHKhg0bTFCSlpZmelGmTZsmzZo1y/O5iYmJEhkZ6Xef3tb7CzJ27FgZM2bMOffPnDlTwsLCpKhkZOU0wU8/zZTQiw7ZSodZs2aJ09EGtIGFdqANFG1wae2Qmpoql6LQp+XGjRvL2rVrJTk5Wb744gu58847ZcGCBfkGKhdj1KhRfj0w2pMSExMjffv2NQm7ReX0mXSR5QvMz3369pGKocHiRBrp6g7Yp08fCQ6mDWgD57aBoh1oA0UbFE07WCMhVyxIKVu2rDRo0MD83K5dO1mxYoXJPXnrrbfOeW5UVJQkJSX53ae39f6ChISEmEtu2kBFubMEn832/hwUVLTvXRIVdfuWRLQBbWChHWgDRRtcWjtcattdcp2U7Oxsv/wRXzosNGfOHL/7NCLLL4elWOuksMwgAAC2UqawwzADBgyQ2NhYOXnypEyePFnmz58vP/30k3l8+PDhUrt2bZNToh577DHp0aOHvPrqqzJo0CCZMmWKmbr89ttvi/0qzhbrpgAAgEsJUg4dOmQCkYSEBKlUqZIp7KYBio5Vqfj4eAkMzOmc6dKliwlknn32WXn66afN1GWd2dOiRQuxA4q5AQBQSoKUd999t8DHtVcltyFDhpiL/VdBLtZNAQAAuTh67R4V4MlFcRGlAABgKwQpnmtCFAAA7MXxQYoVpZCTAgCAvTg+SPH2pBCjAABgKwQpnmt6UgAAsBeCFE+UQowCAIC9EKR4rglSAACwF4IUzzXDPQAA2AtBijXcU9wbAgAA/BCkeK7pSQEAwF4IUjzXxCgAANiL44MUK0qhLD4AAPbi+CCFsvgAANgTQYrnmpwUAADshSCFYm4AANgSQYrnmp4UAADshSDFc02MAgCAvTg+SMmZ3VPcGwIAAHw5PkhhuAcAAHtyfJASSFl8AABsyfFBioWeFAAA7MXxQQqJswAA2BNBiueasvgAANgLQQo5KQAA2BJBiuc6O5swBQAAOyFIoScFAABbcnyQYmF2DwAA9uL4IMUa7qErBQAAeyFI8UQppKQAAGAvBCmea4Z7AACwF4IUzzUhCgAA9kKQ4h3uIUwBAMBOCFKsH4hRAACwFccHKRZ6UgAAsBfHBymBVjE3YhQAAGzF8UGKhZ4UAADsxfFBCrN7AACwJ4IU73APYQoAAHZCkOK5puIsAAD2QpDiuaYjBQCAEhykjB07Vjp06CAVK1aUGjVqyODBg2Xbtm0FvmbSpEkSEBDgdwkNDRW7oJgbAAClIEhZsGCBjBgxQpYuXSqzZs2SzMxM6du3r5w+fbrA14WHh0tCQoL3snfvXrGLAE/KLCEKAAD2UqYwT/7xxx/P6SXRHpVVq1ZJ9+7d832d9p5ERUWJnZE4CwBACQ5ScktOTjbXVapUKfB5p06dkri4OMnOzpa2bdvKiy++KM2bN8/3+enp6eZiSUlJMdfac6OXoqLvZQ33ZJ7NKtL3Lkmsz+3Uz69oA9rAQjvQBoo2KJp2uNT2C3BdZBeCBhw33HCDnDhxQhYtWpTv85YsWSLbt2+XVq1amaDmlVdekYULF8qmTZskOjo6z9eMHj1axowZc879kydPlrCwMClK/90cKNuSA+X3DbKkQ3V6UwAAKCqpqakybNgwc/7X1I8rFqQ8/PDD8sMPP5gAJb9gI7+oqmnTpjJ06FB5/vnnL7gnJSYmRo4cOXJRH7Kgbbl5/BzZmhwoL93cQm5qU0ucSNtBc4z69OkjwcHB4kS0AW1goR1oA0UbFE076Pm7WrVqFx2kXNRwzyOPPCLfffed6REpTICi9EO2adNGduzYke9zQkJCzCWv1xb1zmIN9wQEBjp6R7xc7VvS0Aa0gYV2oA0UbXBp7XCpbVeo2T3a6aIByrRp02Tu3LlSt27dQv+HWVlZsmHDBqlZs6bYAWXxAQCwp0L1pOj0Y80L+frrr02tlMTERHN/pUqVpFy5cubn4cOHS+3atU1NFfXcc89Jp06dpEGDBiZ/5eWXXzZTkO+77z6xE2b3AABQgoOUiRMnmuuePXv63f/+++/LXXfdZX6Oj4+XwMCcDprjx4/L/fffbwKaiIgIadeunSxevFiaNWsmdhDoLeZW3FsCAAAuOki5kN6G+fPn+91+/fXXzcWuKIsPAIA9OX7tHgtl8QEAsBfHBynW7B5CFAAA7IUgxXNN4iwAAPZCkGL1pBCjAABgKwQpnmtyUgAAsBeCFM81MQoAAPbi+CDFQk8KAAD24vggxcpJAQAA9kKQ4rmmJwUAAHshSKEsPgAAtkSQ4rmmIwUAAHshSPFcM9wDAIC9EKSQOAsAgC0RpHius0lKAQDAVghSPNeEKAAA2IvjgxQrSiEnBQAAe3F8kMLsHgAA7IkgxXPtIkoBAMBWCFIo5gYAgC0RpHiuXaTOAgBgKwQpnmt6UgAAsBeCFE+UQkoKAAD24vggxULiLAAA9uL4IMVqAEIUAADsxfFBireYG0kpAADYiuODFMriAwBgTwQpnmvK4gMAYC8EKczuAQDAlghSPNfM7gEAwF4IUjzX5M0CAGAvBCnWcA+pswAA2IrjgxRrXg89KQAA2Ivjg5RAEmcBALAlxwcpFhJnAQCwF8cHKTmze4p5QwAAgB/HBynWcE9mdnZxbwoAAPDh+CAlJMh9fTr9bHFvCgAA8OH4IKWcJ0g5mUaQAgCAnTg+SAkt474mSAEAwF4cH6SUC3JnzJ5MyyzuTQEAAD4cH6SEMtwDAEDJD1LGjh0rHTp0kIoVK0qNGjVk8ODBsm3btvO+burUqdKkSRMJDQ2Vli1byowZM8QuyjHcAwBAyQ9SFixYICNGjJClS5fKrFmzJDMzU/r27SunT5/O9zWLFy+WoUOHyr333itr1qwxgY1eNm7cKHbqScnIypa0zKzi3hwAAODh6Ue4MD/++KPf7UmTJpkelVWrVkn37t3zfM348eOlf//+8uSTT5rbzz//vAlwJkyYIG+++abYJUixelNCg33uAAAAJSNIyS05OdlcV6lSJd/nLFmyREaOHOl3X79+/WT69On5viY9Pd1cLCkpKeZae270UlT0vbSYW/myQXI6I0uOnzojlUOdl6ZjtWlRtm1JQxvQBhbagTZQtEHRtMOltt9FBynZ2dny+OOPS9euXaVFixb5Pi8xMVEiIyP97tPben9BuS9jxow55/6ZM2dKWFiYFLVg0XyUAPlx7gKJqyCOpT1cTkcb0AYW2oE2ULTBpbVDamqqFEuQorkpmleyaNEiKWqjRo3y633RnpSYmBiT/xIeHl5k/49GeNrw1SpVkBOHT0urdldL1/pVxWmsdujTp48EBweLE9EGtIGFdqANFG1QNO1gjYRc0SDlkUceke+++04WLlwo0dHRBT43KipKkpKS/O7T23p/fkJCQswlN22gy7GzhJdzv+eZTJejd8bL1b4lCW1AG1hoB9pA0QaX1g6X2naFSsBwuVwmQJk2bZrMnTtX6tate97XdO7cWebMmeN3n0Zler9dVAxxx2pMQwYAwD7KFHaIZ/LkyfL111+bWilWXkmlSpWkXLly5ufhw4dL7dq1TV6Jeuyxx6RHjx7y6quvyqBBg2TKlCmycuVKefvtt8UuKnhq46dQdRYAANsoVE/KxIkTzYyenj17Ss2aNb2Xzz77zPuc+Ph4SUhI8N7u0qWLCWw0KGndurV88cUXZmZPQcm2V1pFT5BCTwoAACW0J0WHe85n/vz559w3ZMgQc7ErhnsAALAf5xUFKbAnheEeAADsgiCF4R4AAGyJIMV3uCednhQAAOyCIMVndg89KQAA2AdBCsM9AADYEkGKGe5xV8RLOcNwDwAAdkGQIiLVKpQ118dTM+RsVnZxbw4AACBIcatSvqwEBQZItkvkyKmM4t4cAABAkOKmAUqNiu4FDRNT0op7cwAAAEFKjsjwUHOdmEyQAgCAHRCkeESGu3tSDp0kSAEAwA4IUjyi6EkBAMBWCFI8Iiu5g5SklPTi3hQAAECQkiOyohWk0JMCAIAdEKR4RHl6UpjdAwCAPRCk5JrdQ08KAAD2QJCSqydF1+9JzWANHwAAihtBikeFkDJSvmyQ+ZkZPgAAFD+ClDx6UxIIUgAAKHYEKT7iqpY313uPphb3pgAA4HgEKT7iqoaZ671HTxf3pgAA4HgEKT7iqriDlD0EKQAAFDuCFB9x1RjuAQDALghSfNTxyUlxuVzFvTkAADgaQYqP2pXLSWCAyJnMLDl8kjV8AAAoTgQpPsqWCZTaEeXMz3sY8gEAoFgRpOQz5EPyLAAAxYsgJZdYzwyfeHpSAAAoVgQpudCTAgCAPRCk5FvQjZ4UAACKE0FKPqXxtSeFacgAABQfgpR8clJOpp2VE6mZxb05AAA4FkFKLuXKBklUuHs1ZPJSAAAoPgQpeYj15KV8vfagzNqcVNybAwCAIxGk5KGOJ0iZtHiP3P/hSlm770RxbxIAAI5DkFJA8qzlk6V7i21bAABwKoKUPER7SuNbvl1/UJLPkEQLAMCVRJCSh071qkqFkDLSu2mkNImqKGmZ2fL9+oTi3iwAAByFICUPkeGhsvpvfeStO9pJ/xZR5r4Ve44V92YBAOAoZYp7A+y8IrJqExthrtfEHy/mLQIAwFkK3ZOycOFCuf7666VWrVoSEBAg06dPL/D58+fPN8/LfUlMTJSS4KroyuZ6z9FUOX46o7g3BwAAxyh0kHL69Glp3bq1vPHGG4V63bZt2yQhIcF7qVGjhpQElcKCpV5192wfpiIDAGDj4Z4BAwaYS2FpUFK5srtXoqRpExMhuw6fNkM+vZqUjOAKAICS7orlpFx11VWSnp4uLVq0kNGjR0vXrl3zfa4+Ty+WlJQUc52ZmWkuRcV6r/O9Z6vaFeXL1WKqz/6hR10JCgyQ0uRC26E0ow1oAwvtQBso2qBo2uFS2y/AdQlL/WpuybRp02Tw4MEFDvNoXkr79u1N4PHOO+/IRx99JMuWLZO2bdvm+RoNYsaMGXPO/ZMnT5awMHc12CspOUPkxbVBkpYVINfHZknv2u4my/a0XCmLWQAAKBKpqakybNgwSU5OlvDwcPsFKXnp0aOHxMbGmmDlQntSYmJi5MiRIxf1IQuK8GbNmiV9+vSR4ODgAp/75eoD8tdpmyQ4KEAWP9XTLER4/YTFElG+rHx2f0cpyQrTDqUVbUAbWGgH2kDRBkXTDnr+rlat2kUHKcUyBbljx46yaNGifB8PCQkxl9y0gS7HznIh7/u7jnHy3uK98mvSKVm8+7jUq1ZBdh9NNZczWSLhoSV/J75c7VuS0Aa0gYV2oA0UbXBp7XCpbVcsxdzWrl0rNWvWlJJEe42spNl5Ww/JtqST3scOnjhTjFsGAEDpVOielFOnTsmOHTu8t3fv3m2CjipVqpghnFGjRsmBAwfkww8/NI//+9//lrp160rz5s0lLS3N5KTMnTtXZs6cKSXNtY1ryFsLdsmCXw9LtQohfkFKk6iiG4YCAAAXEaSsXLlSevXq5b09cuRIc33nnXfKpEmTTA2U+Ph47+MZGRnyxBNPmMBFk15btWols2fP9nuPkqJdXISEh5aR46mZ8tnKfd77D5xIK9btAgCgNCp0kNKzZ08pKNdWAxVfTz31lLmUBmWCAs2Qz9drD8rJtLPe+xnuAQCg6LHAYCHd3Db6nPtmb06S/v9eKPO2HSqWbQIAoDQiSCmkaxpUO+e+7YdOydbEkzLplz3e+7KtIioAAOCiEKQUklabHdQy75lJuraPBiffrjsozf/xk0xelpObAwAACocg5SL867et5MHu9eT9uzv43Z98JlN2Hz0t7yzaLWcys+TpaRtk1d7jxbadAACUZAQpF6FCSBkZNbCp9GhY/ZzH1sSfkP3HUr23R3+z6QpvHQAApQNByiUIzGPRHh3qOXo6w3t748Fk+WLVfun3+kLZlphTAA4AABSMIOUSDe0YK+WCg2Rkn0bmthZ6U62jK0lslTDR2do67KMVaj9dTo4KAAAXiiDlEr0wuIWs+ltvGXZ1rISUyWnONrER0j4uwvyccTbbXC/bfSzP91i//4S88/MuST+bdYW2GgAA+yNIKYIhn7CyZUyZ/L8OaOK9/6qYytLWE6RYtiammOTa3J6cul7++f0WeeqL9VdkmwEAKAkIUorQnZ3ryKBWNaV25XLSvVF1aV/HP0jRoZ/Ve49L/NFU+WjpXtPDcvx0hnexQq1k+936g8W09QAAlPCy+Ci4V2XC0DbeVZMrlQs2PSzHTqdL29gIWbn3uCzdfVT+9eNWU/wt4cQZc7+v0d9slm4Nq5vXWiX3I8LKSlJKmny+cp+M6NVAyofwawMAlH6c7YqYBie+hd8+vq+jJKdmyr7jZ0yQolVp0z05Kv/7eZf0alzD/Hxz29qybt8J2Xn4tIydsUXG3txSvt+QII9+usYELcFBATJ7yyETvNzROU60oK1OhQYAoLTiLHeZNYkKN9ctM86a5FjtQVEaYJxKPyszNyeZ213qV5Pfto2WYe8skykr9snuI6e9ibY6Y6h82SDz87r9J+TT8fGSmeWS2SN7SDnP/QAAlDbkpFwhmlyrFWqjI8qZnJXPHuwkVcqX9T7eoU6EdGlQTV68qaXpgck9E+h0hnvmz5wth2TP0VQ5cOKMLNzunu4MAEBpRE/KFVSzUjnT+6EjQiFlguTzBzvJne+tkJqVQk1NFaVTmVvUDpelu45KTESY/LzjiN8aQNZQkfppU6L0ax5l1gvS5Qw1uAEAoLQgSLnCQoNzhmca1KgoC57saYIL31yWVtGVzUXpY/ktVDh7c5IcO50hN0xYJFXLl5WpD3WRsj61WtShk2lyy8TFcn2rWvJU/5wp0gAA2B3DPcWsTFCgX4CSW+f6Vb09JNaMH0tK2ll5bMoa2X/8jKzbnywfL917zuv/O2+n7Dt2Rv47f+dl2HoAAC4fghSbqxgaLGNvail/6t1Ibmhdy3v/0I4x5vrn7Ue8942fs930nPjSBFyLDgsBAFBSEKSUALd2iJHHejeUxlEVze2GNSrIs4OamSRcFREWLE2iKppqtprjokNAloTkM96ffRc+BADA7ghSSpDrW9eSfs0j5Ym+jU1Bt1eHtJbI8BD5U59G8ubv25nCcVsSUqT7S/Pk8xX7JDXjrKm7YklM9u9lAQDAzkicLUE0J+WtO9p7b19dr6ose7q39/Yn911tclS0FsuzX2+UqhXKSpbPEI/2qrSMrmTK8a/ce0yqVwiRhpHu3hkAAOyGnpRSRIeDZjzaTRpFVjCBiC5a6CsxJU12HT4l1/xrrgz73zK56b+L5WSa/4KHZzKyZNqa/fL9+oRzHgMA4EqiJ6UUrh90c9toGffDVr+kWZWQnCb/mbtDDp1MN7e14u2KPcekY1xl+Wh7oGyfs0MqlisrY3/Yah6PqVJOvvtjN++sIpfLVeBMJAAAihI9KaXQTW1qe39uHV1JnuzX2Py8Jv64fLvOvcpys5rucv1Ldh6VtxbukpVHAmXC/F2y3KfSrU5dfnb6RhOc6OW2t5dKz5fnSVqmu/qt2nv0tExZHs/MIQAoZmvij8uhlIJzD0vasZogpRSKDA+VB7vXk3ZxEfK/4e0lxlPNdumuY3I22yUd61aRB3vUM/dNW3NA3vslp77Koh3uKc1/vLaBqc+iQc27i3bLweQ0U6pfS/JvTkjxPn/E5NXy1682yJer9+e7PZoXs+fIaRPoAADOT4+Xz0zbICM/X3tBgcWcLUlmCP+3by4xw/Z50RmgvV9bIEPeXFxijscEKaXUqIFN5cuHu0iN8FBTdt/X/d3qSed6Vc3PR05leNcF8i27r0NGowa4K9S+MGOLTJy/w/ucHUmnzPXGA8my8UCKt0R/ft5euEt6vjJfvl7r7sUBAKdKScuUf/24VfYdSy3weToB4pNl8fLV6gPehWnzo73bY77dbH6OP5Yqr83aJmezcpZQsYyfvV12HTktK/Ycl21JBb+nXRCkOEBUeE6QUrdaebmuSY1zgpcmlXJ26OCgAImJKCf3XlNXbmkbLRpwf7w0pzT/jsOnZNPBZBn9zSbvfVpUTqc86x/e2n0n/P7/WZvdAQwLIgJwutdm/ioT5++UAeN/zvNx7eE4kZohP2xI8N6nszHzojWxlu06Kg99vMoEJ+XLupdd+d/Pu6XNc7Pky1U5Pdzbk07KR0v3eG8v/DXnePzp8nh5Y94Ov6F8uyBx1iHDP5Z7rqlrkmuVFoT7YtU++Wu/RvK/bxfK1mT3c+pULW/K9SsNVHIP5bzz8y7TO2IpGxRoemC6jptrSvVnu1zy1cNdpE1shGRmZcvGg+7elm3n+TYAAKXdqr3HvRMXDp44I7Uqu4tyWnQJk5d/2uZ338o9x+W37aLl0+X7zJfLgS1rmnzAgeN/9vaE65fLf9/WxuSlfLRkr5xMPytPTF0nHy3dK01rVpRfdhyVzKycIZ6Fvx6RB7rXN4HO2BlbzLG7RsUQGdLeXc3cLghSHEAXHbytQ4zsPZoqv20b7b1/UKua5pKZmSlRPn8n9aqX9/6sO7f2vviV1/cZyuzeqLrUq1ZeJi3eI8dTc6Ysz9lyyAQpWxNOmunQavuhU3LPpBWm8u2U+ztJOU/UDwBOoT3OluHvLZeHe9SXm9vWNvmCQQEB8kkea7B9s+6grNxzzOQG6gTLX/5yrbzw/RYToFQpX1baxlY2C8g2iqwofZpFyp/7NpbXZv0qE+btMD3bVu92bJUweem3rcwkCM0/fG3mNtmSeNIEKE1rhpthfrshSHGIcbe0KvDxqLCcyKN+9Qren3XK8cCWUfLGvJ0SUsbdY2KZ+lBn6VCnivk2oIsc1qocKqkZWfLFqv3yy84j8pvEmjJ7S5L3+RqszN16yPw8fe0BGdoxtog/JQAUneQMkQ+W7JV+LWp5JyBcCh1O0ckHlh2HTpneDq1J9frs7abWlQYilv7No+RHT76fdb8Ov4/6aoMs+PWwmdzw2QOdzinKqb3lf+7XWAa3qS2/Jp2UxTuPmC+pY25obr501qoUat7v/+bm5BrqY9ZitnZCkAKjYrB7DSDtDfENUtSt7WNMTsr1rWv65aa0jY0w19pd+c6d7kq4+4+nmiBlTfwJ6f/vvMdc1QeL95jeHQ2CZm9OkhkbE0y5/9q5uj4B4FLoTJfAQJGQMoXruZ295ZCMWR0kWa5t8u2GJPl6RNc880f0i5kuU3L8dIYZwtFgRvM/9Lio9/vaefiUme2oQ+RtYiubGZPque82mx5qqwTEVTGV5YN7OkrFkDLS9V9zTY0r9czApmYigwYo6o5OcQVWDW9Qo4K56PCQr3/c0FymrtxnFrA9cOKMdK1fzcz6tCOCFHjpukDfb0iUTvXdM38scVXLy9q/9zEBhRWkaDCRV9QdHRFmuh99FzlUOtZpFZFTmq3+5eoD0qFOhDzy6WpJy8w2466fPdhJalYql2cXqY6hdq5f1Vtc7mKkn82Sf/2wTa5rWkO6Nqh20e8DwP6SUzNl4P/9bHqBf/pTdwn25Nr5Hg/yC17Gz90pWS73MW7dvhNmNmOL2pW8j2t5hldnbjM9Iy/d0kr+9/Mu01vxUI96poeiRe1wGXNDC/l5+2GT56HTiLUulRWEfPZgZ5On1+/fC/2G0K1hdOs49/ffNDN5JaNvaG4CHx3GOZOZZSqL/6W/ewZmYfVrHmUuJQGze+D1/A3NZPXf+uTZm2FVmn1lSGuz+vLbw9vl+z49G1X3/nxD61rmD7J/iyi/hRLVn6euM/P6NUBRmp3+0o/bvENDmt1uzffXRDLNYNfkXOtbxMXQKXjv/bJbbn9nmd/9OmSlXbsA7O3oqXQTMFyI/y7YYXoKdNqtfglSOnNGv/RoEcqmf/vR9PzmlpSSZr5IBYhLutRz9zD85j+LZMLc7WYGo/ag6JRfa+hGpxRrzl1GVrZ3CEXLM9wycbH8e/Z2s+hrt5fmeZcqaRTl7q3W4R0tuKn0uFrG88Wvh88xdEDLmjL5/k4m36RCSBl55NoGZtX7/97e1hF5ffSkwE/ubxq5aYa5XgqiqzJrL8t93eqZP0Lrm8iHS/aaP67Xb21txkS10q32uJQLDjLjoU99ud5Mi9NvHDodbvyc7aaonA4DWYGJdqc+/91m6TGyh9//qQee93/ZY5732HUNpVeTGnluW17ToPU9f/PGEgnICpJbb3DJxffTALjcHvholZkhoyu/+375UfqlRntNNCdDgxM9Jljmbk2SOtXCpO/rC80wipXj8Z+5202vhQ7FPNCtnnmtdbyJKS/yh571ZPEu9zDMKzN/Nbkjelw6ciqnZ1gnA/iqXjFEDnt6jnWl+qSUnOeqxlHuit/qsd4N5akv1svo65ub2ZC63ZoIm58RvRqYi1MQpKDI6Zjsy0Na+93XOqayfPvINe5vC0GBptjc7VfHmQODPl+zzkd/u8n8ses3mDlb3Qm3+u1HvzntOpwzu0hfo4GFfquwPPH5OpnjScp9ccYW6dm4ep7rDGmpf98kttDgINl0IFlOpmnGfYAZkooNKXtZ2gVA4eiCqCHBQd7e3fijqd4pvI9OWSNd61c1q8E/1KO+ea7WHtFhXK20/dy3m0yPbHhoGTN7RY8POoNG/9bdf+9umkvy+JQ13qm8n6/c5z3eNI1wScc6EXJP17pmyq++bvmeY6Z3RGkeh/aqaHE0a8hc80eeu7GF6Y1JOZNpak3tOnJKwkODzbpoOqvGt7f52iaRsvLZPlewVUsWghRcMS093ZqW2Kph5mK5um4VmbftsHy7/qBs8tRW0RL81gFAx2BTzpw1qzlvPpjiTfTSRLSlu9xjvUq7XXXKnU6B9pWYnGbKQlv0IKL5NlYdF2vIKbZa/oloAC4P/dKgPbnaC6u5JI99tkbmbztsctwWPtXLDPEs9uR0KA1A9HihF62grb0f6Z4ZhHe9v9wUmNThk/fu6mCm3GrgoTkjSr8U6XFE30MDGIsuzOqraeVs82Xn79c3M7f1y1H7f87yDlHrMUt7YfQYpSM1Y29uZbY3twY13MeU1393FQu1FhJBCmxDvwHpAUerMVo0qPhm3QHzc/s6VUwXauLmNNlwIFmqlA+WP3yyWq6uW9V8C9KeFU2I1fL7+m1Is9515pDWDdAse/0G5OvgCXeQotVzLTqVGkDR0gTV4e8ul2oVQ2TC0DbnnKT1b/DWN5dIcJlAGXyV1gzJNgGK0iHhRz9d4y1foHTIWXNDdGaNHgt0dozWGLFogKLu7VbXHDf02KJBjH6h0V7d6X/oYo4ZmpumQ9G+NLDR+lEVQ4IkLmC332N6jOnTLMq7UKsee/TLk65v1qV+tTwDlNwIUAqHIAW2oRnt4kks8zVjg7tOQMc6VUxPx6zNSeZb1Yrdx+TXpFPmojRBV2uvaJBiTYPWoaPfd4qVfw5uKfO35RzkVGKKOyDZ5Fl/SO09lmoOWk1qVvTL+tcuZq35UiWsrNzXrS4HGjieDsNqkOBbP0TLsL+5YKf839A2pjiYRafWWtNtG0dWNLNcbusYIzdeVdvkoP1t+kb3cEtGlikMabEKSfoGKOqpfo3N0h7aO9rrlfneISD1u/YxkpqZZXpXftfBXT3137+7Sr7fkGCOH3qf/v1qwNEmprI3SNFKrjpUc2eXOvK33zQzRS5nzPAPUtRNbWqZIEWDGZ1GrF+Aloy6rkjbFjkIUmAbmr3+ZL/G3pLQVsEhpV3AnerlTD/W1ZtzxwmabKbdr/ot6Pv1Cd5FuXSBruGd63i//WiFXM321wOSdjHrWkSWiQt2m4t+23rvzvZStUKI/LLjiPz+3WWmiJL5f+IizArTuc3beshk2+t2AsVJT7xapVQrj17oDBAdytDgQYdRw8q4K0trT2a14OBzZsJpIDJlxT5z+/s/XiO1I8qZXshnpm8wQyFa6fSNYW29r9GS7BadQquW7DpqekOqlA+R1fEnJKxskJlSq7lp+remf8sP9awvd7+/wvtaXXdM//40QFFRlULlDz3ry6ue99Sct3G3tDznS0RE+bLy+05x53zm1jHuIWh9+hcPdzFffvT/KEiPRjXkvmvqSlzVsHPqoKDo0cKwFc1a71SvimxLPCWn08+awkVKv/3oAcm3NoseyPTbjCbDqTZxEebg9OqQ1iZhbf3+ZDNzSMeeNaNf6XtrgKEVdBNOpJmDknYB53WQv3vSClPA6aUft3oDFGvFZ10WXQ9sesDSg7p+k9Pn6+atH93Pm9T7+Yp9ZnGwF25q6Z05pauTau+PLjlAjwwuBz3Ra0+iJpzqel0XYsw3m2Tqqv0mWBh3U3P5ak+gPDluvvxnaBszDdYKxB/8aJWZaut93bebzf6vtTssMzclmp4WDfKVVjz1ZRWO1Nky1p+0zsrTXgy9/f7iPaY3QwuRhQYHmsBHq6++ece5pQ/u717PBEw6K6Zbw2qF+pvq1rC6mYmjXyy0rS6kmKQeg579jTtHBZcfdVJgO+3iqsiwq2OleS2faXrXNTTXekDRdS70IKc9Iq/e6p5FpAc27bpVOmPnw3s6yvJnrpOHe9b3e++HezbwFovTAkl3eb6lRYW7D6a+NMh56adtss4T7Px1gLtwkhZt0kXANNC5/Z2lJk9G6yQojXeWepL7NClvzLeb5POV+01vjOXZ6RtNgalv1+escloaTFuzX6avcecPwZ8maWvC5MXQgmHXvjLf9DoozbPS3ggdJrHoY1rDQ3tDNLjfsN+dZ7Vst3tf1GB8ns9wp57QfYdIdOqrVX5dh3DG/vir/JwYaL4APPzJavP+2uuo+64GKFqEUXs9lc5WsQIUnUmjf5e6kF27f86WIW8ulsemrDF/S2pIu2hpWKOC6bWw/p70YwxqWVPu71bP3L6jcx2Z+0RPUzhN/5a1WqrGHcM7n9sTovQ5/77tKlNbRMseFEa1CiGy4pne8n+3XVWo18HGPSkLFy6Ul19+WVatWiUJCQkybdo0GTx4cIGvmT9/vowcOVI2bdokMTEx8uyzz8pdd911KdsNB9Dqsnog1B4H35WcX7vV/4CSnpktIcGBUjksJ2lNv01pTsmQ9tGycu9xSc/Mkm6Nqkv3htX8xrf1oK5DO38f2Fhuectd4E2/uWk1Rs1tsZJ47+5axwRHmv1vnWustYy0botVzlppsbjJy+NNjow1rVHX6OjZuIaZxmh1k2sxKc2zyXK5SvxyAHoS/tNn68y3zGub1jDTLUs63TeOHk/zzsy4WDM2JJgE72ub1JAJw9pIWNnzH3Z3HDop932w0uRpfbx0rzmR6xDLM4OayuOfrTXbpsObuk9p8KC9eLqPHTmVIb2bRnp7FxfvOGpmulhJqFpbpG+zSLnjnWVmyFN7SbS4ou6/Or22fNkgycx2ecuwW16Zuc0EFxrc6DDsh/dcbfb/T5fHm2EerTvy+UOdpValcvLjpgT5y5cbzOvcM/PcwZAGL76lCer3qGASTXW7R/Zp5F2dPbcXb2ppHtdq1vnRNcS0jPzFokezFPWknD59Wlq3bi1vvPHGBT1/9+7dMmjQIOnVq5esXbtWHn/8cbnvvvvkp59+upjthYPogUOHf7SOQEFu7eBOwMuLnhD0QPz28PZmnQt9Tx028i3X//mDnaRl7ZxeG52eqMl3Fi3G9IdeDaRGxVBT6lppRr8m0Cod+1dWoKHTJDUQssbe1XZPcq8WqbPorIXrJyySPq8tkD0+q0yr9ftPmF4a36nVhaH1JHRYqiCab6Dd93kNdxWWFaTpe+n/fbnpUIKu/6S9BhdKews06VpP6kp7NnynpFvFwLRi6MHTIs9+vVl6v7bQDO1Z9TS0Fkdu+j6aMHrb20tMr4FvD4fV06B0n/jj5DV5bpu+h/aQLNp+xGyD9r5pNVMtgGi93Zer98uU5ftMgKKsCqra06InevXZin0mSLCcTD/rDVCUtpkmsGqAop7+aoP5fWm7KM3n6t6wut/fhzVbxloS48n+7jwXDSoe6dXA9GLqWjCaKFspLFiGtIsxQ67jb7vKXAZ4Cq5pomxuui7Y0wObmt6Q/OhjBQUoKN0K3ZMyYMAAc7lQb775ptStW1deffVVc7tp06ayaNEief3116Vfv36F/e+BS6bf9iw6G0h7XDI9dQ+scWodo9bEOK2r8MygZt4cE03A1SXSdbxck+Z0yMcy8fdt5YYJv+T5f24/dFLWxB/3K8FtJfYq7UbX8fbHp6w1t63Vo0+kZsr3j3YzY/rfrU8wRaV0nN6XJjseOplmen+sg70WutJaMe8Mby+9m7mDPO2y16mSNcJDzHIFOmQ1eVm8Cd6eH9zC+35aUC+yUhl5deavZjs+uvdqiT92WqpXCPXWtdGTqs6W0CDhdx1ivcMK1kn5r1+tNyew65pGmqGAouxZ0f9b21m/1evqsY9c29AEBjqjw7fAX25//3qjfLp8nyn8pUMNExfsNMsw6An1Fk8V5aenbTBJ2ZHlgiTpjDvI++Ona0zxwX/9sNV8Fl11Vqe1WhZuP2JWsrU82L2+NPMZqvQNQOduO+QtRKjb/MmyvSZQqlOtvDziCWC0pyLcZ30qDQJ0zRbtsdCpthbNddJ6Ih8s3usdutAqqL6LgFoe6F5P3vl5l0lWPZ1x1vu+GsQ8+/VG+dWzL2ovzPHUDO/+90ivejL2h1+9a3HpsOeAFjmL1d3WMdZMB9YCjd7tDQzwtqeyZvDQWQFbJs4uWbJEevfu7XefBifao5Kf9PR0c7GkpLiniOqUML0UFeu9ivI9SyKntUP5YJEmWhgu7azccXW0d796qGmWHA+Lkcd61ZOsrLPy7h1tzZRkHSKy2uam1lHmYvUaWAmAXepXkaaR5U0FzF92HpVWtcNlvc/UZi1O9+TUdeZb8Q2tasrsrYfM2L/viV17VHJ3s+vrnvh8jXy52t1bM2dzknz+QEdz0lJ6grvz/eUm/6VyuWDTI3RXlzhZt/+EN0+kR0P3CXXC3F/ls5XunJEfNybI+v0p3tycW9rUlEbVy8m6owHy2Lj50rleFVniKQU+5puN8t2GRPNZl4/qZU44o7/bIp+ucAdcHeIqyTKfHh+rINZDH6+WHg2rybI9x+SrBztJtssl+0+cMT1OujzCxZq25qAJUKyhlAe71ZH/zN0pbyzYJZPubGeSo/OiAYrSYZMHromV/3oCzH9+v1l6NqxiEps1QFFJZ3LOqPp70imyltdmbpOXbmlhZpppULhil39S6Mo9RyRQss1woU6htXo5lA4VztuSaHrWFu86Kiv3un9P2rYWndFmzWob0bOeaavgoAB56BN3AGvRIHTa6niTI6L78x+vrS8jPs0Jlq5tXF3menpRHu1ZV3YdOimztx725of888bm8rdvNpulKKy8rM51K5sgShPSs7OzpXv9CJkZV1kWeWbndK1fRYIk2y+oV5nZOftyaeG04+LlaodLbb8A18Vmc3m648+Xk9KoUSO5++67ZdSoUd77ZsyYYYaAUlNTpVy5c8fiR48eLWPGjDnn/smTJ0tYGN1+uHRZLpGz2SIhl7g+17Q9gbIgIUAeapotTSq75FSmyOE0kchyIm9vDZIaoS5ZcThAssV90qsQ7JKnW2eZx/acct8XU94l+0+LuCRAygS4pFmES46nB8ihNJH0rJyTZXiwS1IyAySynEuG1c+Sw2kBEhwo8v6v/h8iONAlmdnu15UNdMkL7bNEZ6H+a12QHEx1368Lpyn9P1Wlsi55oEmWfPBrkBxKy/8r7786nJW1xwLk0505/+dt9bJkyq6CG7J+RZfsOun+/wLFJY+3yJIVhwOlfiWXtKla8CHo2/hA2XI8wGxfxWCRMWuCJDkjwNsmz7fPkhfXas9HgLSpmi13NfI/gapDZ0ReWOv+TlahjEuurZUt38TnbPM1kdmmXb/ck//n6BaVLYsSteXc/3fNci55tEWWfLg9ULacCJSwIJekZgVI1RD9/Yn3d25pVClbfk0OlJAgl9/v1VenGtmy9JC7V0Lf5+9tc07+u0+K/LgvUGIriPySFCCnzwZ4f9fXx2ZJ79ou2XIiQFYd0W1wb++XuwOla2S2NKikgZfIe9uCJPFMgESXd8mfW2bJF7sDZVGS+/97uGmW2YeVvo8G1M0jXDL7QIB862kr/V13jrz04UE4R2pqqgwbNkySk5MlPDynh7FET0HWgEYTbX17UjThtm/fvhf1IQuK8GbNmiV9+vSR4Fy1AJyEdrj4NuiX7c5ryKvS5G9vcF/3fn2R6ZFRj/VpIkM6x8mGgM2yx9MT8f4D3eTYqQx595c9ckvb2tKrsTsn4M0Fu+TV2e4cljuujpH7u9WVW99eJokp6fL6RvefbsVQvT4rt7arLTe0ril3Tlolvl9yM7IDJKx+O+nWoJqMXDbXhCXazZ/tWYJe1a9eXnYePi0vr3e/Z7ngQDmT65uyJbZ1Z3njK+1VyKnMuylda8bk9BrlZedJ9/+nibU6e/W9neXkxJlM+TlJpOGNzeTW9jnDA28t3C2frdwvk+5qZ2aJPL70F9MDsTEgTrrXrSbJy9aZBE9NStagrWG77pK0ZLF57ZqjgRJ0sIqZRnvTVbXMew27OkayTK+VO0dIA4llJ/TLTroMbl1Tpq9LMCdq9/R2/xNw3aphUq96eRnStrapZqx5KrptKuFMgHxztIYkpOtQSabc3a2+vDF/lxxNPzcA0dyO6zvEmN+nFaDc0yVOBrSIlLs/WG16L3S13XE3t5Aery40n7dn89oycGDOMJwa4bl+8OM1ppfECkb/PKSn6V0bmOv/vTXX7btcLjlwIk0qhwWbIacuqRnyxNQNpibJg91zpir38fl7iE5KlW89SeWP3NLLL4m9NOO4WDTtYI2EXKzLHqRERUVJUpJ7fNOitzXYyKsXRYWEhJhLbtpAl2NnuVzvW9LQDoVvA31m6HkWJEw7m/Nt+PZOdSQ4uIw0r11ZZMV+k39Qv0a4NIgMkI71cxIW1Q1touU/83aZaddPDWgqFUODTX7Ib99c4k34tBZK6964hlzTKNIMTenSAr7F8L5YfVCiKpc3w1P6XlrXRYvdKa0rMWFoW5PDYq38OqJnfWkTV8WcxO54d5nf2ibj5+6U+GNnTFD2+6tjTZLpBs+wVp2qYd6l6y1WMrEOz+iQxbibW5n8DQ1QLDrk0LlBdalXvYLJMdETveZ+TFubaF5n9fV+ufqATF3lHo65q2sdk+yp9WYme4IGizWEsnC7e4ji799sMSdli/YQ6Kq0uj1jb2kt9WtUNMXAtH00cTTLTMd1H7P6NI8yiZ2WMTe2kC4NqpmckUc+We0dEtPhEf3d6rZbdBbNTE9Cav3qFaRNnBb5cwedVcuXlacHNTO5HDpzZewPW+ThXg3MulGaF6KfrXfTqHz3RU0WX7TzqBnm0+fHVb/wL291a+TsrzUqBctH93XK97n6/+t2a/GyKhXKSnRV561rxXHx0trhUtvusgcpnTt3NsM7vjQq0/sBJ3j0uobyzLSNJtnWmn6qdSF+2Jhg1inJb/qjriv04+PdTO6DBihK1yOa+afuJkFSK3Hqqs3KqnI7uE1tb5Dyr9+2kjvfW25ua1KlalW7krSNjfAGKZrgqTMydPrm0h2HZPrcpXJv1zgpFxrifV/rRKuWek7Kd3ep45ccqnS1V6vypwZD3/3xGikbFCifrdxn8lR01Wudxq11ZjRpWGdKVS0fYoIjTejVk60mg1o1N75avd8U4lOal2ElGmtzacKz5u9okJJXoqhOidWkUK1AqgmnmoCss7ZMXoxnfabW0ZXNLJVHrm1gkqBTNAm3VwOZOG+7N0jRabe+NAfFmkmmCbjW59Xfi5UnpDQAuqNznLfttDemZe2cBTb7t4jyJptqsTWd4m7tB1r7Z+P+ZDMFPz/9W9SUTWMiTUKr9bu9XDQRluJlKC6FDlJOnTolO3bs8JtirFOLq1SpIrGxsWao5sCBA/Lhhx+axx966CGZMGGCPPXUU3LPPffI3Llz5fPPP5fvv/++aD8JYFNDO8SaGhlWETmrTPcnBXyDtWjvQm7a3a4XLXinS8brdGjrRNW3WZSp+1K9Qohc06CaqYGhCZxaSdRaiVrra1ia18o5cWol3qRIl99MDZ02qidafY2WLrdoj4NVQdeiAdJrs381PR8aVFjDAg90q2fqWGjdGD0Ra7CmRfKeu6G5mR2kQcony+LNxZeVPNqzcXV5ZUhrU/lXt0EXjNQpqXrS1yUPLDocNmfrIdPe+hwtVvab1jVl6sr9Jli64apaMuqrDbL/uPs1HTyraOs2+VZlbeyT1KvBR36Gd6njDVI0adl3Fs3/hrc3n9mibaXBoE7T3ZKQYtrKl2+gqjOhtLfmfPQ9nTL0AucqdJCycuVKU/PEYuWO3HnnnTJp0iRT4C0+Pudgo9OPNSD505/+JOPHj5fo6Gh55513mH4Mx9Bvor4BSlG595q6cvRUhvlWbtGeAS3lb9GegR82JpphAdUqupLpAdGppNpj4fvtPi9ao2b32IHmhK/DTFavgAZPOstHcz909otO19aF5mqGu4eYdBE538/vu9aRrkj7teckrHn7vr0kVi+IrgVj3ffHaxuaIOyl3+YUAlPXNYk06zxZs6RubhttpotbdFqv8l2zRQuKWXTYKy9aPFBpjkru6d6+tIfr779pZpZu0Ho+1sJ3GqhYQePgq2qZysJWjZA3f9/W9OT4BjAAijBI6dmzZ4HlnTVQyes1a9bkXcQIwMXRISDf+iZ50Z6A6X/oKm8t3GkCFV1OXuvCaE0WLYimU2TPR7/lN/SpumoNLWnwoe+v6xxp5VxVt3p5E6Q09ymOd7731hVzp67cZ4aDtIaHDolpYTM9+WtvUF6LOSqt2TLj0W7yzqJdpi0uZFpzfU/QoTmy+b1vVHio3F4/Szq0a1Ng3RWlPTC63oy1ppT2QvkOv4y7pZU8PaipKQRoDeHpBUAJnt0DoOhoz8n429r43afrnBSGDlVYdKE3i85E0iBFh3/UqAFNTSEwXW+lMKtf+/aAWImx2lNhFaLLj/aW/HNwywv+v9rHRZgZUZ3rVS2wwFzHGi4Z2DLncxbEd9HL3DSHpaBqqgAKRpAC4IJoIqxWrdX1ZHyHkzQ3xuop0EXh9HKpNN9CZ7AUtRrhoWZBuYICCwD2QZAC4ILkFYDo8IYVoJQU9GwApXiBQQAAgCuBIAUAANgSQQoAALAlghQAAGBLBCkAAMCWCFIAAIAtEaQAAABbIkgBAAC2RJACAABsiSAFAADYEkEKAACwJYIUAABgSwQpAADAlkrEKsgul8tcp6SkFOn7ZmZmSmpqqnnf4OBgcSragTZQtIEb7UAbKNqgaNrBOm9b5/FSGaScPHnSXMfExBT3pgAAgIs4j1eqVKmwL5MA18WGN1dQdna2HDx4UCpWrCgBAQFF9r4a4Wngs2/fPgkPDxenoh1oA0UbuNEOtIGiDYqmHTTE0AClVq1aEhgYWDp7UvSDRUdHX7b314Z38k5ooR1oA0UbuNEOtIGiDS69HS6mB8VC4iwAALAlghQAAGBLjg5SQkJC5B//+Ie5djLagTZQtIEb7UAbKNrAHu1QIhJnAQCA8zi6JwUAANgXQQoAALAlghQAAGBLBCkAAMCWHB2kvPHGG1KnTh0JDQ2Vq6++WpYvXy4l0dixY6VDhw6mIm+NGjVk8ODBsm3bNr/n9OzZ01Tr9b089NBDfs+Jj4+XQYMGSVhYmHmfJ598Us6ePev3nPnz50vbtm1NpneDBg1k0qRJYhejR48+5zM2adLE+3haWpqMGDFCqlatKhUqVJBbbrlFkpKSSlUb6P6cuw30op+7tO4HCxculOuvv95UtNTPM336dL/HdW7A3//+d6lZs6aUK1dOevfuLdu3b/d7zrFjx+T22283xaoqV64s9957r5w6dcrvOevXr5du3bqZ44VW4HzppZfO2ZapU6eafU6f07JlS5kxY4bYoR10/ZW//OUvZpvKly9vnjN8+HBTyft8+8+4ceNKTDucb1+46667zvl8/fv3L1X7wsLztEFexwe9vPzyy/bcD1wONWXKFFfZsmVd7733nmvTpk2u+++/31W5cmVXUlKSq6Tp16+f6/3333dt3LjRtXbtWtfAgQNdsbGxrlOnTnmf06NHD/MZExISvJfk5GTv42fPnnW1aNHC1bt3b9eaNWtcM2bMcFWrVs01atQo73N27drlCgsLc40cOdK1efNm13/+8x9XUFCQ68cff3TZwT/+8Q9X8+bN/T7j4cOHvY8/9NBDrpiYGNecOXNcK1eudHXq1MnVpUuXUtUGhw4d8vv8s2bN0tl7rnnz5pXa/UC38ZlnnnF99dVX5rNOmzbN7/Fx48a5KlWq5Jo+fbpr3bp1rhtuuMFVt25d15kzZ7zP6d+/v6t169aupUuXun7++WdXgwYNXEOHDvU+rm0UGRnpuv32283f2aeffuoqV66c66233vI+55dffjHt8NJLL5l2efbZZ13BwcGuDRs2FHs7nDhxwvxOP/vsM9fWrVtdS5YscXXs2NHVrl07v/eIi4tzPffcc377h+9xxO7tcL594c477zS/a9/Pd+zYMb/nlPR9YcZ52sD3s+tFz4EBAQGunTt32nI/cGyQon+gI0aM8N7Oyspy1apVyzV27FhXSacnKt05FyxY4L1PT06PPfZYgTt2YGCgKzEx0XvfxIkTXeHh4a709HRz+6mnnjJBgK/f/e53JkiyS5CiB5e86EFa/0CmTp3qvW/Lli2mnfSAXVraIDf9ndevX9+VnZ3tiP0g90FZP3dUVJTr5Zdf9tsXQkJCzIFV6QFUX7dixQrvc3744Qdz4D5w4IC5/d///tcVERHhbQP1l7/8xdW4cWPv7VtvvdU1aNAgv+25+uqrXQ8++KDrSsvr5JTb8uXLzfP27t3rd3J6/fXX831NSWqH/IKUG2+8Md/XlLZ9QS5gP9D2uPbaa/3us9N+4MjhnoyMDFm1apXp9vVdH0hvL1myREq65ORkc12lShW/+z/55BOpVq2atGjRQkaNGmWW37bo59buuMjISO99/fr1M4tLbdq0yfsc3zaznmOnNtNufO3mrFevnumy1aELpb9v7fL23X7thoyNjfVuf2lpA9/9/OOPP5Z77rnHb2FOJ+wHlt27d0tiYqLf9uo6Ijq86/t712799u3be5+jz9djwrJly7zP6d69u5QtW9bvM+uw6vHjx0tcu1jHCd0v9LP70m59HRJt06aNGQLwHeorDe2gQ5U6jNm4cWN5+OGH5ejRo97HnLYvJCUlyffff2+GtHKzy35QIhYYLGpHjhyRrKwsvwOx0ttbt26VkkxXjH788cela9eu5iRkGTZsmMTFxZkTuI4l6vi07lBfffWVeVwP5Hm1h/VYQc/RE9iZM2fMeH9x0hOP5kbowSchIUHGjBljxkw3btxotl3/oHIfkHX7z/f5rMdKQhv40rHoEydOmHF4J+0Hvqxtzmt7fT+PnrR8lSlTxgT5vs+pW7fuOe9hPRYREZFvu1jvYSean6W/+6FDh/otGvfoo4+aXCP97IsXLzZBrP4tvfbaa6WiHTT/5OabbzafYefOnfL000/LgAEDzIkzKCjIcfvCBx98YHIZtU182Wk/cGSQUpppgqSelBctWuR3/wMPPOD9Wb8paxLhddddZ/5Q69evL6WBHmwsrVq1MkGLnpA///xzW504r5R3333XtIkGJE7aD1Aw7VG89dZbTULxxIkT/R4bOXKk39+QBvYPPvigSc4vDeXhb7vtNr/9Xz+j7vfau6J/B07z3nvvmR5nTWy1637gyOEe7erWqDn3zA69HRUVJSXVI488It99953MmzdPoqOjC3yunsDVjh07zLV+7rzaw3qsoOfoNzE7BgHaa9KoUSPzGXXbdfhDexby+52XpjbYu3evzJ49W+677z5H7wfWNhf0t67Xhw4d8ntcu7Z1lkdR7Bt2OqZYAYruH7NmzfLrRclv/9C22LNnT6lqB4sOC+v5wHf/d8q+8PPPP5te1PMdI4p7P3BkkKJRYbt27WTOnDl+wyR6u3PnzlLS6DciDVCmTZsmc+fOPacbLi9r16411/pNWunn3rBhg98fqHUQa9asmfc5vm1mPceubabTBrWHQD+j/r6Dg4P9tl//QDVnxdr+0tQG77//vum21qnETt4P9G9BD4q+26vDUppf4Pt71+BV85Ys+nekxwQriNPn6NROPcn7fmYdWtSu7ZLQLlaAonlbGsBqvsH56P6h+RjWEEhpaAdf+/fvNzkpvvu/E/YFq6dVj4utW7cWW+8HLgdPQdYM/0mTJpmM7gceeMBMQfad1VBSPPzww2aK5fz58/2mjKWmpprHd+zYYaaT6bTb3bt3u77++mtXvXr1XN27dz9n6mnfvn3NNGadTlq9evU8p54++eSTZmbMG2+8Yavpt0888YRpA/2MOv1Np1zq9Fmd7WRNQdap2XPnzjVt0blzZ3MpTW1gzVTTz6nZ9r5K635w8uRJM11aL3pIe+2118zP1qwVnYKsf9v6edevX29mM+Q1BblNmzauZcuWuRYtWuRq2LCh37RTnRGkUy7vuOMOM+VSjx/aBrmnXJYpU8b1yiuvmHbR2WZXcgpyQe2QkZFhpl5HR0eb36vvccKaobF48WIzo0Mf1+moH3/8sfndDx8+vMS0Q0FtoI/9+c9/NrP5dP+fPXu2q23btuZ3nZaWVmr2hZPn+XuwphDrNuvMvdzsth84NkhRWt9BD+ZaL0WnJOu8+JJId8S8Llo7RcXHx5sTUZUqVUxgpvP+9QTjWx9D7dmzxzVgwAAz311P7nrSz8zM9HuO1tu46qqrTJvpCc76P+xAp8HWrFnTbFvt2rXNbT0xW/Sk9Ic//MFMndM/qJtuuskcpEtTG6iffvrJ/P63bdvmd39p3Q90W/La/3W6qTUN+W9/+5s5qOrnvu66685pm6NHj5oTUYUKFcx067vvvtsc7H1pjZVrrrnGvIfuXxr85Pb555+7GjVqZNpFp2l///33Lju0g56U8ztOWDV0Vq1aZaaI6hee0NBQV9OmTV0vvvii3wnc7u1QUBvolzYNvvWEqydLnWarNYNyfzEt6fvCvPP8PSgNJvTvW4ON3Oy2HwToP4XrewEAALj8HJmTAgAA7I8gBQAA2BJBCgAAsCWCFAAAYEsEKQAAwJYIUgAAgC0RpAAAAFsiSAEAALZEkAIAAGyJIAUAANgSQQoAALAlghQAACB29P+EQSXRDvnxTgAAAABJRU5ErkJggg=="
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "execution_count": 19
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "# 推理",
   "id": "587c7b7fd8bf6be6"
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": [
    "temperature的作用是控制模型的随机性，即模型输出的概率分布。\n",
    "\n",
    "文本生成时，temperature越高，模型越倾向于输出高概率的字符，即模型更加随机；temperature越低，模型越倾向于输出低概率的字符，即模型更加确定。\n",
    "\n",
    "选取的不是最大值，而是按照一定概率分布进行采样，这样可以让模型更加符合真实的情况。"
   ],
   "id": "bc47742805ae5c4b"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-06T04:43:49.227633Z",
     "start_time": "2025-03-06T04:43:49.218634Z"
    }
   },
   "cell_type": "code",
   "source": [
    "#下面的例子是为了说明temperature\n",
    "my_tensor = torch.tensor([0.4, 0.6])  #这里是logits\n",
    "\n",
    "probs1 = F.softmax(my_tensor, dim=-1)\n",
    "print(probs1)"
   ],
   "id": "845a5916648fb6a7",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([0.4502, 0.5498])\n"
     ]
    }
   ],
   "execution_count": 20
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-06T04:43:54.491177Z",
     "start_time": "2025-03-06T04:43:54.486743Z"
    }
   },
   "cell_type": "code",
   "source": [
    "my_tensor = torch.tensor([0.2, 0.3])  #现在 temperature是2\n",
    "\n",
    "probs1 = F.softmax(my_tensor, dim=-1)\n",
    "print(probs1)"
   ],
   "id": "fb1b127f4c380075",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([0.4750, 0.5250])\n"
     ]
    }
   ],
   "execution_count": 21
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-06T04:45:13.683456Z",
     "start_time": "2025-03-06T04:45:13.648187Z"
    }
   },
   "cell_type": "code",
   "source": [
    "import torch\n",
    "\n",
    "# 创建一个概率分布，表示每个类别被选中的概率\n",
    "prob_dist = torch.tensor([0.1, 0.45, 0.35, 0.1])\n",
    "\n",
    "# 使用 multinomial 进行抽样\n",
    "# num_samples 表示要抽取的样本数量\n",
    "num_samples = 5\n",
    "\n",
    "# 抽取样本，随机抽样，概率越高，抽到的概率就越高\n",
    "samples = torch.multinomial(prob_dist, 1, replacement=True)\n",
    "\n",
    "print(\"概率分布:\", prob_dist)\n",
    "print(\"抽取的样本索引:\", samples)\n",
    "\n",
    "# 显示每个样本对应的概率\n",
    "print(\"每个样本对应的概率:\", prob_dist[samples])"
   ],
   "id": "a7d14a1c8eee7486",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "概率分布: tensor([0.1000, 0.4500, 0.3500, 0.1000])\n",
      "抽取的样本索引: tensor([1])\n",
      "每个样本对应的概率: tensor([0.4500])\n"
     ]
    }
   ],
   "execution_count": 22
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "### 文本生成",
   "id": "50b502eb99c85e28"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-03-06T05:01:40.982357Z",
     "start_time": "2025-03-06T05:01:39.655966Z"
    }
   },
   "cell_type": "code",
   "source": [
    "def generate_text(model, start_string, max_len=1000, temperature=1.0, stream=True):\n",
    "    input_eval = torch.Tensor([char2idx[char] for char in start_string]).to(dtype=torch.int64, device=device).reshape(1,\n",
    "                                                                                                                      -1)\n",
    "    hidden = None\n",
    "    text_generated = []\n",
    "    model.eval()\n",
    "    pbar = tqdm(range(max_len))\n",
    "    print(start_string, end=\"\")\n",
    "    # no_grad是一个上下文管理器，用于指定在其中的代码块中不需要计算梯度。在这个区域内，不会记录梯度信息，用于在生成文本时不影响模型权重。\n",
    "    with torch.no_grad():\n",
    "        for i in pbar:  #控制进度条\n",
    "            logits, hidden = model(input_eval, hidden=hidden)\n",
    "            # 温度采样，较高的温度会增加预测结果的多样性，较低的温度则更加保守。\n",
    "            #取-1的目的是只要最后，拼到原有的输入上\n",
    "            logits = logits[0, -1, :] / temperature\n",
    "            # 使用 multionomial来采样\n",
    "            probs = F.softmax(logits, dim=-1)  #算为概率分布\n",
    "            idx = torch.multinomial(probs, 1).item()  #从概率分布中抽取一个样本,取概率较大的那些\n",
    "            input_eval = torch.Tensor([idx]).to(dtype=torch.int64, device=device).reshape(1, -1)  #把idx转为tensor\n",
    "            text_generated.append(idx)\n",
    "            if stream:\n",
    "                print(idx2char[idx], end=\"\", flush=True)\n",
    "    return \"\".join([idx2char[i] for i in text_generated])\n",
    "\n",
    "\n",
    "# load checkpoints\n",
    "model.load_state_dict(torch.load(\"checkpoints/text_generation/best.ckpt\", map_location=\"cpu\"))\n",
    "start_string = \"All: \"  #这里就是开头，什么都可以\n",
    "res = generate_text(model, start_string, max_len=1000, temperature=0.5, stream=True)"
   ],
   "id": "fffdbc4275e69f42",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "  0%|          | 0/1000 [00:00<?, ?it/s]"
      ],
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "1a1d2f80f94f4c979091589139e17828"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "All: I,\n",
      "Hark you, sir, that you must not see,\n",
      "The shepherd is come to slaughter'd here, to see my head to be a power\n",
      "The clouds that forehold have me speak.\n",
      "\n",
      "QUEEN ELIZABETH:\n",
      "What do you persuade him, his son we may dead.\n",
      "\n",
      "DUKE VINCENTIO:\n",
      "And let him not seen their children's enemy, that thou wast for some other blood and bosom, which we did, I'ld not be so fitted for this action.\n",
      "\n",
      "PROSPERO:\n",
      "Soft! I'll say so well I love his death!\n",
      "The which I do prove so straight for the people,\n",
      "As he shall parting this?\n",
      "\n",
      "Provost:\n",
      "Away with her shame;\n",
      "And see him do lie lived\n",
      "Your files, I would not die to-morrow;\n",
      "The father was the man may kiss the gentleman,\n",
      "Where is my love, he did before thee a courteous excuse\n",
      "As may be so triumph to hear thee shall stand for state:' whose legs did state to see:\n",
      "The very blood that so sticks of such remedies.\n",
      "\n",
      "KING RICHARD III:\n",
      "How now, my lord,\n",
      "If you shall be holy as I, with wings shall suffer that changed again;\n",
      "And I will speak against him and all the rest: it is "
     ]
    }
   ],
   "execution_count": 23
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
